import redis
import time
import json

r = redis.StrictRedis(host='127.0.0.1', port=6379)

def redis_string_urllib_raw_data(url, data):
	global r
	r.set('urllib:csdn:raw_pages:%s' % url, data)


def redis_hash_urllib_raw_data(url, data):
	global r
	r.hset('urllib:csdn:raw_pages:%s' % url, 'content', data)

def redis_list_urllib_raw_data(url, data):
	global r
	r.rpush('urllib:csdn:raw_pages:%s' % url, data)

filename = 'spider_csdn.net_data_file.dat'
fp = open(filename, 'r')
count = 0 
while True:
    data = fp.readline()
    if not data or len(data) <= 0:
        print None
        break 
    count += 1
    decoded_items = json.loads(data)
    if 'URL' in decoded_items and 'CONTENT' in decoded_items:
        print count, decoded_items['URL']
        #redis_hset_urllib_raw_data(decoded_items['URL'], decoded_items['CONTENT'])
        #redis_list_urllib_raw_data(decoded_items['URL'], decoded_items['CONTENT'])
        redis_string_urllib_raw_data(decoded_items['URL'], decoded_items['CONTENT'])
        time.sleep(0.01)

fp.close()
